# -*- coding: utf_8 -*-

import uuid
from datetime import datetime
import re
#import DB

import config

class LogParser:
    
    #dictionary for storing sessions
    #key - session id
    #value - 
    sessions = {}

    #list of IPs deemed non-useful (automated pollers, search robots)
    ip_blacklist = []

    def __init__(self, apache_log_file):
        self.apache_log_file = apache_log_file
        
    def remove_robot(self):
        self.ip_blacklist.append("220.181.94.231")
        self.ip_blacklist.append("65.55.25.140")
        self.ip_blacklist.append("221.226.14.142")
        self.ip_blacklist.append("218.94.125.35")
        self.ip_blacklist.append("218.94.74.8")
        self.ip_blacklist.append("218.94.125.34")
        self.ip_blacklist.append("221.226.30.42")
        
        robot_list = open(config.output_path+"robot_list","r")
        for line in robot_list.readlines():
            self.ip_blacklist.append(line)
        
        no_robot_log = open(config.output_path + "no_robot_log","w+")
        
        for line in self.apache_log_file:
            
            #whether or not the requested file is robots.txt
            #if it is, we add the ip to the blacklist
            
            if line.url.find("robots.txt") != -1:
                self.ip_blacklist.append(line.ip)
            
            remove = False
            #if ip is in blacklist
            if self.ip_blacklist.count(line.ip) > 0:
                remove = True
            if line.ip.find("61.135.24") >= 0:	
                remove = True
            if line.ip.find("203.208.60") >= 0:
                remove = True
            if line.ip.find("61.135.248") >= 0:
                remove = True
            
            if not remove:
                no_robot_log.write(str(line))
        no_robot_log.close()
    
    def clean_log(self): 
        purged_log = open(config.output_path + "purged_log","w+")
        for line in self.apache_log_file:
            remove = False

            #no images, CSS or JS files
            remove = remove or line.url.find(".png") != -1
            remove = remove or line.url.find(".gif") != -1
            remove = remove or line.url.find(".jpg") != -1
            remove = remove or line.url.find(".pjpeg") != -1
            remove = remove or line.url.find(".png") != -1
            remove = remove or line.url.find(".x-png") != -1
            remove = remove or line.url.find(".js") != -1
            remove = remove or line.url.find(".css") != -1
            remove = remove or line.url.find(".ico") != -1
            remove = remove or line.url.find(".swf") != -1
    
            #no error pages
            remove = remove or line.url.find("/errorleht") != -1
            remove = remove or line.http_response_code == '302'
    
            #no HTTP OPTIONS and POST
            remove = remove or line.http_method.find("OPTIONS") != -1
            remove = remove or line.http_method.find("POST") != -1
    
            #no home directories
            remove = remove or line.url.find("/~") != -1
            
            #no VerifyCaptchaServlet
            remove = remove or line.url.find("VerifyCaptchaServlet") != -1
            
            if not remove:
                if config.remove_query_string:
                    self.__format_url(line)
                purged_log.write(str(line))
                
        purged_log.close()
    
    def generate_sql_scr(self):
        purged_log_sql = open(config.output_path + "purged_log_sql","w+")
        
        for line in self.apache_log_file:
            line.time = line.time.split()[0]
            line._time = line.time[line.time.find(":")+1:]
            line.date = datetime.strptime(line.time, '%d/%b/%Y:%H:%M:%S')
            sql = "insert into logitem(ip,agent,userid,url,date,size)"+\
                                 "values('%s','%s','%s','%s','%s','%s');\n" % (line.ip, line.ident, line.http_user, line.url, str(line.date), str(line.http_response_size))
            purged_log_sql.write(sql)
            
        purged_log_sql.close()
        
    def __format_url(self, line):
        
        #format URLs so that equivalent URLs would always be same
        
        #strip parts starting with ? from urls
        qmark_index = line.url.find("?")
        if qmark_index != -1:
            line.url = line.url[0:qmark_index]
        #strip slash from end of url
        
        if len(line.url) > 0 and line.url[len(line.url) - 1] == "/":
            line.url = line.url[0:len(line.url) - 1]
        
        #if len(line.url) > 0 and line.url[0] == "/":
        #    line.url = line.url[1:]
        
        if len(line.url) == 0:
            line.url = "avaleht"